import requests
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
import time
import os
import random

# 网站根路径
baseUrl = 'http://www.netbian.com'
# 首页链接
homePageUrl = '/mei/index.htm'
# 伪造请求头
ua = UserAgent()
headers = {'User-Agent':ua.random}

# 统计图片数量的全局变量
count = 0


# 请求网页
def request_page(url):
    global headers
    resp = requests.get(url=url, headers=headers)
    resp.encoding = resp.apparent_encoding

    bs = BeautifulSoup(resp.text, 'html.parser')
    return bs
# 爬取单页数据
def get_page_one(url):
    global count

    bs1 = request_page(url)
    # 1.获取下一页链接
    # 页面中上一页和下一页的元素特征全部相同 需要通过文本对下一页的元素进行筛选
    nextpage = bs1.select('.page>a.prev')
    for a in nextpage:
        if a.text == '下一页>':
            nextpage = a
            done = False
        else:
            done = True


    # 2.获取当前页图片并存储
    lis = bs1.select('.list>ul li')
    # 删除第三张图片的html结构(第三张图片html结构与其他图片不同)
    del lis[2]

    # 遍历每张小图的li
    for li in lis:
        # 获取高清大图链接
        a_href = baseUrl + li.find('a')['href']
        # 进入大图展示页面
        bs2 = request_page(a_href)

        # 获高清大图链接
        img = bs2.select_one('.endpage>.pic img')
        img_name = img['alt']
        img_src = img['src']
        # 发送请求 获取图片二进制数据
        img_resp = requests.get(url=img_src, headers=headers)
        # 随机睡0-.1秒
        # time.sleep(random.uniform(0,0.01))
        # 写入文件
        with open(f'./images/{img_name}.jpg', 'wb') as f:
            f.write(img_resp.content)
            f.close()
            print(f'{img_name}.jpg ------ 下载完成')
            count += 1

    if done == True:
        print('爬取完成')
        return True
    next_page_url = baseUrl + nextpage['href']
    # 随机睡0-.01秒
    time.sleep(random.uniform(0,0.01))
    # 递归调用爬取数据
    get_page_one(next_page_url)

def main():
    startTime = time.time()

    # 获取当前文件所在文件夹绝对路径
    dir = os.path.dirname(__file__)

    if not os.path.exists(os.path.join(dir,'images')):
        # 如果当前文件根目录不存在图片文件夹则创建存储图片的文件夹
        os.mkdir(os.path.join(dir,'images'))
    # 调用函数开始爬取
    get_page_one(baseUrl + homePageUrl)
    endTime = time.time()
    runTime = endTime - startTime
    print(f'用时{runTime}秒', f'共{count}张图片')


if __name__ == '__main__':
    main()



